{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 290,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "import spacy\n",
    "import pandas as pd\n",
    "import numpy as np\n",
    "import nltk\n",
    "from nltk.corpus import wordnet as wn\n",
    "from nltk.tokenize.toktok import ToktokTokenizer\n",
    "import re\n",
    "from bs4 import BeautifulSoup\n",
    "from contractions import CONTRACTION_MAP\n",
    "import unicodedata\n",
    "\n",
    "nlp = spacy.load('en', parse = False, tag=False, entity=False)\n",
    "tokenizer = ToktokTokenizer()\n",
    "stopword_list = nltk.corpus.stopwords.words('english')\n",
    "stopword_list.remove('no')\n",
    "stopword_list.remove('not')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                                              review sentiment\n",
      "0  One of the other reviewers has mentioned that ...  positive\n",
      "1  A wonderful little production. <br /><br />The...  positive\n",
      "2  I thought this was a wonderful way to spend ti...  positive\n",
      "3  Basically there's a family where a little boy ...  negative\n",
      "4  Petter Mattei's \"Love in the Time of Money\" is...  positive\n"
     ]
    }
   ],
   "source": [
    "dataset = pd.read_csv(r'movie_reviews.csv')\n",
    "\n",
    "print(dataset.head())\n",
    "\n",
    "reviews = np.array(dataset['review'])\n",
    "sentiments = np.array(dataset['sentiment'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 316,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "def strip_html_tags(text):\n",
    "    soup = BeautifulSoup(text, \"html.parser\")\n",
    "    stripped_text = soup.get_text()\n",
    "    return stripped_text\n",
    "\n",
    "\n",
    "def expand_contractions(text, contraction_mapping=CONTRACTION_MAP):\n",
    "    \n",
    "    contractions_pattern = re.compile('({})'.format('|'.join(contraction_mapping.keys())), \n",
    "                                      flags=re.IGNORECASE|re.DOTALL)\n",
    "    def expand_match(contraction):\n",
    "        match = contraction.group(0)\n",
    "        first_char = match[0]\n",
    "        expanded_contraction = contraction_mapping.get(match)\\\n",
    "                                if contraction_mapping.get(match)\\\n",
    "                                else contraction_mapping.get(match.lower())                       \n",
    "        expanded_contraction = first_char+expanded_contraction[1:]\n",
    "        return expanded_contraction\n",
    "        \n",
    "    expanded_text = contractions_pattern.sub(expand_match, text)\n",
    "    expanded_text = re.sub(\"'\", \"\", expanded_text)\n",
    "    return expanded_text\n",
    "\n",
    "\n",
    "def remove_accented_chars(text):\n",
    "    text = unicodedata.normalize('NFKD', text).encode('ascii', 'ignore').decode('utf-8', 'ignore')\n",
    "    return text\n",
    "\n",
    "\n",
    "def remove_special_characters(text):\n",
    "    text = re.sub('[^a-zA-z0-9\\s]', '', text)\n",
    "    return text\n",
    "\n",
    "\n",
    "def lemmatize_text(text):\n",
    "    text = nlp(text)\n",
    "    text = ' '.join([word.lemma_ if word.lemma_ != '-PRON-' else word.text for word in text])\n",
    "    return text\n",
    "\n",
    "\n",
    "def remove_stopwords(text, is_lower_case=False):\n",
    "    tokens = tokenizer.tokenize(text)\n",
    "    tokens = [token.strip() for token in tokens]\n",
    "    if is_lower_case:\n",
    "        filtered_tokens = [token for token in tokens if token not in stopword_list]\n",
    "    else:\n",
    "        filtered_tokens = [token for token in tokens if token.lower() not in stopword_list]\n",
    "    filtered_text = ' '.join(filtered_tokens)    \n",
    "    return filtered_text\n",
    "    \n",
    "\n",
    "\n",
    "def normalize_corpus(corpus, html_stripping=True, contraction_expansion=True,\n",
    "                     accented_char_removal=True, text_lower_case=True, \n",
    "                     text_lemmatization=True, special_char_removal=True, \n",
    "                     stopword_removal=True):\n",
    "    \n",
    "    normalized_corpus = []\n",
    "    \n",
    "    for doc in corpus:\n",
    "        \n",
    "        if html_stripping:\n",
    "            doc = strip_html_tags(doc)\n",
    "            \n",
    "        if contraction_expansion:\n",
    "            doc = expand_contractions(doc)\n",
    "            \n",
    "        if accented_char_removal:\n",
    "            doc = remove_accented_chars(doc)\n",
    "            \n",
    "        if text_lower_case:\n",
    "            doc = doc.lower()\n",
    "            \n",
    "        special_char_pattern = re.compile(r'([{.(-)!}])')\n",
    "        doc = special_char_pattern.sub(\" \\\\1 \", doc)\n",
    "        \n",
    "        if text_lemmatization:\n",
    "            doc = lemmatize_text(doc)\n",
    "            \n",
    "        if special_char_removal:\n",
    "            doc = remove_special_characters(doc)  \n",
    "            \n",
    "        # remove extra whitespace\n",
    "        doc = re.sub(' +', ' ', doc)\n",
    "        \n",
    "        if stopword_removal:\n",
    "            doc = remove_stopwords(doc, is_lower_case=text_lower_case)\n",
    "            \n",
    "        normalized_corpus.append(doc)\n",
    "        \n",
    "    return normalized_corpus\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 317,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "n = normalize_corpus(reviews[0:10000])"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python [conda root]",
   "language": "python",
   "name": "conda-root-py"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
